
!module list

Currently Loaded Modulefiles:
 1) cuda/11.8.0              5) openmpi/4.1.5-cuda   9) sparsehash/2.0.3       
 2) nccl/2.18.5-1-cuda       6) intel-mkl/2020.4    10) libjpeg-turbo/2.1.3    
 3) cudnn/8.7.0.84-cuda      7) magma/2.7.1-cuda    11) pytorch-gpu/py3/2.1.1  
 4) gcc/8.5.0(8.3.1:8.4.1)   8) sox/14.4.2          
>


from idr_pytools import display_slurm_queue, gpu_jobs_submitter, search_log
from dlojz_tools import controle_technique, compare, GPU_underthehood, plot_accuracy, lrfind_plot, imagenet_starter, turbo_profiler
MODULE = 'pytorch-gpu/py3/2.1.1'
account = 'for@a100'
name = 'pseudo'   ## Pseudonyme à choisir


display_slurm_queue(name)


#!scancel -u $USER


#!scancel <jobid>


#jobid = ['2088207']


%cat {search_log(contains=jobid[0])[0]}


%cat {search_log(contains=jobid[0], with_err=True)['stderr'][0]}


s1 = "dlojz_da_2.py"
s2 = "./solutions/dlojz_da_2.py"
s1 = "mixup.py"
s2="solutions/mixup-solution.py"
compare(s1, s2)


bs_optim = 512
image_size = 224


import os
import torchvision
import torchvision.transforms as transforms
import torchvision.models as models
import torch
import numpy as np
import matplotlib.pyplot as plt

transform = transforms.Compose([ 
        transforms.RandomResizedCrop(image_size),  # Random resize - Data Augmentation
        transforms.RandomHorizontalFlip(),  # Horizontal Flip - Data Augmentation
        transforms.RandAugment(5, 9),       # Random Augmentation 5: n operations, 9 : magnitude 
        transforms.ToTensor()               # convert the PIL Image to a tensor
        ])
    
    
train_dataset = torchvision.datasets.ImageNet(root=os.environ['ALL_CCFRSCRATCH']+'/imagenet',
                                                  transform=transform)
train_dataset

Dataset ImageNet
    Number of datapoints: 1281167
    Root location: /gpfsscratch/idris/sos/commun/imagenet
    Split: train
    StandardTransform
Transform: Compose(
               RandomResizedCrop(size=(224, 224), scale=(0.08, 1.0), ratio=(0.75, 1.3333), interpolation=bilinear, antialias=warn)
               RandomHorizontalFlip(p=0.5)
               RandAugment(num_ops=5, magnitude=9, num_magnitude_bins=31, interpolation=InterpolationMode.NEAREST, fill=None)
               ToTensor()
           )


%%time

train_loader = torch.utils.data.DataLoader(dataset=train_dataset,    
                                           batch_size=4,
                                           shuffle=True)
batch = next(iter(train_loader))
print('X train batch, shape: {}, data type: {}, Memory usage: {} bytes'
      .format(batch[0].shape, batch[0].dtype, batch[0].element_size()*batch[0].nelement()))
print('Y train batch, shape: {}, data type: {}, Memory usage: {} bytes'
      .format(batch[1].shape, batch[1].dtype, batch[1].element_size()*batch[1].nelement()))

for i in range(4):
    img = batch[0][i].numpy().transpose((1,2,0))
    plt.imshow(img)
    plt.axis('off')
    plt.show()

X train batch, shape: torch.Size([4, 3, 224, 224]), data type: torch.float32, Memory usage: 2408448 bytes
Y train batch, shape: torch.Size([4]), data type: torch.int64, Memory usage: 32 bytes

CPU times: user 2.7 s, sys: 377 ms, total: 3.08 s
Wall time: 3.14 s


command = f'dlojz_da_1.py -b {bs_optim} --image-size {image_size} --test'
n_gpu = 1
jobid = gpu_jobs_submitter(command, n_gpu, MODULE, name=name,
                    account=account, time_max='00:10:00')
print(f'jobid = {jobid}')

batch job 0: 1 GPUs distributed on 1 nodes with 1 tasks / 1 gpus per node and 8 cpus per task
Submitted batch job 887754
jobid = ['887754']


jobid = ['887754']


display_slurm_queue(name)

             JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)
            887754    gpu_p5     tutu  ssos938  R       2:32      1 jean-zay-iam05

 Done!


controle_technique(jobid)

Train throughput: 2079.58 images/second
GPU throughput: 2430.28 images/second
epoch time: 616.25 seconds
-----------
training step time average (fwd/bkwd on GPU): 0.210675 sec (8.9%/99.2%) +/- 0.074190
loading step time average (IO + CPU to GPU transfer): 0.035528 sec +/- 0.241765


turbo_profiler(jobid)

>>> Turbo Profiler >>> Training complete in 48.483087 s


import os
import torchvision
import torchvision.transforms as transforms
import torchvision.models as models
import torch
import numpy as np
import matplotlib.pyplot as plt

transform = transforms.Compose([ 
        transforms.RandomResizedCrop(image_size),  # Random resize - Data Augmentation
        transforms.RandomHorizontalFlip(),  # Horizontal Flip - Data Augmentation
        transforms.ToTensor()               # convert the PIL Image to a tensor
        ])
    
    
train_dataset = torchvision.datasets.ImageNet(root=os.environ['ALL_CCFRSCRATCH']+'/imagenet',
                                                  transform=transform)
train_dataset

Dataset ImageNet
    Number of datapoints: 1281167
    Root location: /gpfsscratch/idris/sos/commun/imagenet
    Split: train
    StandardTransform
Transform: Compose(
               RandomResizedCrop(size=(224, 224), scale=(0.08, 1.0), ratio=(0.75, 1.3333), interpolation=bilinear, antialias=warn)
               RandomHorizontalFlip(p=0.5)
               ToTensor()
           )


from mixup import mixup_data


%%time

train_loader = torch.utils.data.DataLoader(dataset=train_dataset,    
                                           batch_size=16,
                                           shuffle=True)
batch = next(iter(train_loader))
print('X train batch, shape: {}, data type: {}, Memory usage: {} bytes'
      .format(batch[0].shape, batch[0].dtype, batch[0].element_size()*batch[0].nelement()))
print('Y train batch, shape: {}, data type: {}, Memory usage: {} bytes'
      .format(batch[1].shape, batch[1].dtype, batch[1].element_size()*batch[1].nelement()))

imgs, targets = batch
imgs, targets = mixup_data(imgs, targets, num_classes=1000, alpha=2)        ## Transformation mixup

for i in range(4):
    img = imgs[i].numpy().transpose((1,2,0))
    plt.imshow(img)
    plt.axis('off')
    plt.show()
    print(f'target : {torch.max(targets, dim=1)[1][i]}, lambda : {torch.max(targets, dim=1)[0][i]}')

X train batch, shape: torch.Size([16, 3, 224, 224]), data type: torch.float32, Memory usage: 9633792 bytes
Y train batch, shape: torch.Size([16]), data type: torch.int64, Memory usage: 128 bytes

target : 932, lambda : 0.6358286738395691

target : 760, lambda : 0.5654819011688232

target : 888, lambda : 0.8834490180015564

target : 24, lambda : 0.7696910500526428
CPU times: user 7.45 s, sys: 262 ms, total: 7.71 s
Wall time: 7.93 s


for alpha in [0.5, 1., 2.]:
    plt.hist(np.random.beta(alpha, alpha, 1000000), bins=50, density=True, histtype='step')
    plt.title(f'alpha={alpha}')
    plt.show()

from mixup import mixup_data


command = f'dlojz_da_2.py -b {bs_optim} --image-size {image_size} --test'
n_gpu = 1
jobid = gpu_jobs_submitter(command, n_gpu, MODULE, name=name,
                    account=account, time_max='00:10:00')
print(f'jobid = {jobid}')

batch job 0: 1 GPUs distributed on 1 nodes with 1 tasks / 1 gpus per node and 8 cpus per task
Submitted batch job 887894
jobid = ['887894']


jobid = ['887894']


display_slurm_queue(name)

             JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)
            887894    gpu_p5     tutu  ssos938  R       2:38      1 jean-zay-iam33

 Done!


controle_technique(jobid)

Train throughput: 1107.83 images/second
GPU throughput: 2524.46 images/second
epoch time: 1156.80 seconds
-----------
training step time average (fwd/bkwd on GPU): 0.202816 sec (6.1%/110.0%) +/- 0.084701
loading step time average (IO + CPU to GPU transfer): 0.259348 sec +/- 0.078801


turbo_profiler(jobid)

>>> Turbo Profiler >>> Training complete in 50.497971 s


command = f'dlojz_da_2.py -b {bs_optim} --image-size {image_size} --test'
n_gpu = 1
jobid = gpu_jobs_submitter(command, n_gpu, MODULE, name=name,
                    account=account, time_max='00:10:00')
print(f'jobid = {jobid}')

batch job 0: 1 GPUs distributed on 1 nodes with 1 tasks / 1 gpus per node and 8 cpus per task
Submitted batch job 887909
jobid = ['887909']


jobid = ['887909']


display_slurm_queue(name)

             JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)
            887909    gpu_p5     tutu  ssos938  R       2:11      1 jean-zay-iam16

 Done!


controle_technique(jobid)

Train throughput: 2188.81 images/second
GPU throughput: 2521.39 images/second
epoch time: 585.49 seconds
-----------
training step time average (fwd/bkwd on GPU): 0.203062 sec (8.6%/110.8%) +/- 0.097770
loading step time average (IO + CPU to GPU transfer): 0.030855 sec +/- 0.063996


turbo_profiler(jobid)

>>> Turbo Profiler >>> Training complete in 44.058701 s


import os
import torchvision
import torchvision.transforms as transforms
import torchvision.models as models
import torch
import numpy as np
import matplotlib.pyplot as plt

transform = transforms.Compose([ 
        transforms.RandomResizedCrop(image_size),  # Random resize - Data Augmentation
        transforms.RandomHorizontalFlip(),  # Horizontal Flip - Data Augmentation
        transforms.ToTensor()               # convert the PIL Image to a tensor
        ])
    
    
train_dataset = torchvision.datasets.ImageNet(root=os.environ['ALL_CCFRSCRATCH']+'/imagenet',
                                                  transform=transform)
train_dataset

Dataset ImageNet
    Number of datapoints: 1281167
    Root location: /gpfsscratch/idris/sos/commun/imagenet
    Split: train
    StandardTransform
Transform: Compose(
               RandomResizedCrop(size=(224, 224), scale=(0.08, 1.0), ratio=(0.75, 1.3333), interpolation=bilinear, antialias=warn)
               RandomHorizontalFlip(p=0.5)
               ToTensor()
           )


from cutmix import cutmix_data


%%time

train_loader = torch.utils.data.DataLoader(dataset=train_dataset,    
                                           batch_size=16,
                                           shuffle=True)
batch = next(iter(train_loader))
print('X train batch, shape: {}, data type: {}, Memory usage: {} bytes'
      .format(batch[0].shape, batch[0].dtype, batch[0].element_size()*batch[0].nelement()))
print('Y train batch, shape: {}, data type: {}, Memory usage: {} bytes'
      .format(batch[1].shape, batch[1].dtype, batch[1].element_size()*batch[1].nelement()))

imgs, targets = batch
imgs, targets = cutmix_data(imgs, targets, num_classes=1000)

for i in range(4):
    img = imgs[i].numpy().transpose((1,2,0))
    plt.imshow(img)
    plt.axis('off')
    plt.show()
    print(f'target : {torch.max(targets, dim=1)[1][i]}, lambda : {torch.max(targets, dim=1)[0][i]}')

X train batch, shape: torch.Size([16, 3, 224, 224]), data type: torch.float32, Memory usage: 9633792 bytes
Y train batch, shape: torch.Size([16]), data type: torch.int64, Memory usage: 128 bytes

target : 492, lambda : 0.9307438135147095

target : 64, lambda : 0.901945173740387

target : 103, lambda : 0.8318518996238708

target : 776, lambda : 1.0
CPU times: user 6.52 s, sys: 242 ms, total: 6.76 s
Wall time: 6.96 s

from cutmix import cutmix_data


command = f'dlojz_da_3.py -b {bs_optim} --image-size {image_size} --test'
n_gpu = 1
jobid = gpu_jobs_submitter(command, n_gpu, MODULE, name=name,
                    account=account, time_max='00:10:00', qos='qos_gpu-dev')
print(f'jobid = {jobid}')

batch job 0: 1 GPUs distributed on 1 nodes with 1 tasks / 1 gpus per node and 8 cpus per task
Submitted batch job 887951
jobid = ['887951']


jobid = ['887951']


display_slurm_queue(name)

             JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)
            887951    gpu_p5     tutu  ssos938 CG       2:33      1 jean-zay-iam22

 Done!


controle_technique(jobid)

Train throughput: 1759.63 images/second
GPU throughput: 2512.19 images/second
epoch time: 728.30 seconds
-----------
training step time average (fwd/bkwd on GPU): 0.203807 sec (7.0%/118.0%) +/- 0.104394
loading step time average (IO + CPU to GPU transfer): 0.087163 sec +/- 0.036166


turbo_profiler(jobid)

>>> Turbo Profiler >>> Training complete in 48.079175 s

mixed_x = x
for i in range(len(mixed_x)): # loop over images
            mixed_x[i,:,x1[i]:x2[i],y1[i]:y2[i]] = x[s_index[i],:,x1[i]:x2[i],y1[i]:y2[i]]

def cut_mask(x1, x2, y1, y2, batch_size, W, H, device=None):
    
    ### TODO
    mask_ext, mask_int = None, None
    
    return mask_ext, mask_int


import torch
import numpy as np
import matplotlib.pyplot as plt
batch_size = 3
W = 32
H = 32


# coordonnee min dans la largeur pour chaque image du batch
x1 = torch.Tensor([10, 5, 23]).int()
# coordonne max dans la largeur pour chaque image du batch
x2 =  torch.Tensor([20, 25, 31]).int()
# coordonnee min dans la hauteur pour chaque image du batch
y1 =  torch.Tensor([5, 10, 0]).int()
# coordonne max dans la hauteur pour chaque image du batch
y2 =  torch.Tensor([10, 22, 20]).int()


# initialisation du tenseur w_int avec les valeurs [0,...,31]
w_int = torch.arange(W).repeat(batch_size,1,1) # batch de vecteurs ligne
# Returns the mask applying ((x1 ⩽ x) and (x ⩽ x2))
w_int = torch.logical_and(w_int >= x1.view(-1,1,1), w_int <= x2.view(-1,1,1)) # vecteurs ligne
# assert w_int has the correct size
assert w_int.size() == torch.Size([3,1,32])


for wx in w_int:
    plt.imshow(wx)
    plt.colorbar()
    plt.show()


# initialisation du tenseur h_int avec les valeurs [0,...,31]
h_int = torch.arange(H).repeat(batch_size,1).unsqueeze(2) # batch de vecteurs colonne
# Returns the mask applying ((y1 ⩽ y) and (y ⩽ y2))
h_int = torch.logical_and(h_int >= y1.view(-1,1,1), h_int <= y2.view(-1,1,1)) # vecteurs colonne
# assert h_int has the correct size
assert h_int.size() == torch.Size([3,32,1])


for hx in h_int:
    plt.imshow(hx)
    plt.colorbar()
    plt.show()

# initialisation du tenseur w_int avec les valeurs [0,...,31]
w_int = torch.arange(W).repeat(batch_size,1,1) # batch de vecteurs ligne 
# Returns the mask applying ((x1 ⩽ x) and (x ⩽ x2))
w_int = torch.logical_and(w_int >= x1.view(-1,1,1), w_int <= x2.view(-1,1,1)) # vecteurs ligne

# initialisation du tenseur h_int avec les valeurs [0,...,31]
h_int = torch.arange(H).repeat(batch_size,1).unsqueeze(2) # batch de vecteurs colonne
# Returns the mask applying ((y1 ⩽ y) and (y ⩽ y2))
h_int = torch.logical_and(h_int >= y1.view(-1,1,1), h_int <= y2.view(-1,1,1)) # vecteurs colonne

w_int = torch.zeros(batch_size,1,W) # vecteurs ligne 
w_int[range(batch_size),0,x1] = 1.
w_int[range(batch_size),0,x2+1] = -1.
w_int = torch.cumsum(w_int, dim=2).bool() # vecteurs ligne

h_int = torch.zeros(batch_size,H,1) # vecteurs colonne
h_int[range(batch_size),y1,0] = 1.
h_int[range(batch_size),y2+1,0] = -1.
h_int = torch.cumsum(h_int, dim=1).bool() # vecteurs colonne


# multiplication des vecteurs colonne "hauteur" h_int par les vecteurs ligne "largeur" w_int
mask_int = h_int*w_int


# visualisation des masques intérieurs pour chaque image du batch
for m in mask_int:
    plt.imshow(m)
    plt.colorbar()
    plt.show()

# multiplication des vecteurs colonne "hauteur" h_int par les vecteurs ligne "largeur" w_int
mask_int = h_int*w_int


# les masques extérieurs sont les complémentaires des masques intérieurs
mask_ext = torch.logical_not(mask_int)


# visualisation des masques extérieurs
for m in mask_ext:
    plt.imshow(m)
    plt.colorbar()
    plt.show()

# les masques extérieurs sont les complémentaires des masques intérieurs
mask_ext = torch.logical_not(mask_int)

# rajouter une dimension en 2e position pour pouvoir traiter des images RGB
    mask_int = mask_int.unsqueeze(1) 
    mask_ext = mask_ext.unsqueeze(1)


def cut_mask(x1, x2, y1, y2, batch_size, W, H, device=None):
    
    mask_ext, mask_int = None, None
    
    # initialisation du tenseur w_int avec les valeurs [0,...,31]
    w_int = torch.arange(W, device=device).repeat(batch_size,1,1) # batch de vecteurs ligne
    # Returns the mask applying ((x1 ⩽ x) and (x ⩽ x2))
    w_int = torch.logical_and(w_int >= x1.view(-1,1,1), w_int <= x2.view(-1,1,1)) # vecteurs ligne 
    
    # initialisation du tenseur h_int avec les valeurs [0,...,31]
    h_int = torch.arange(H, device=device).repeat(batch_size,1).unsqueeze(2) # batch de vecteurs colonne
    # Returns the mask applying ((y1 ⩽ y) and (y ⩽ y2))
    h_int = torch.logical_and(h_int >= y1.view(-1,1,1), h_int <= y2.view(-1,1,1)) # vecteurs colonne
    
    # multiplication des vecteurs colonne "hauteur" h_int par les vecteurs ligne "largeur" w_int
    mask_int = h_int*w_int

    # les masques extérieurs sont les complémentaires des masques intérieurs
    mask_ext = torch.logical_not(mask_int)
    
    # rajouter une dimension en 2e position pour pouvoir traiter des images RGB
    mask_int = mask_int.unsqueeze(1) 
    mask_ext = mask_ext.unsqueeze(1)
    
    return mask_ext, mask_int


%%time

train_loader = torch.utils.data.DataLoader(dataset=train_dataset,    
                                           batch_size=16,
                                           shuffle=True)
batch = next(iter(train_loader))
print('X train batch, shape: {}, data type: {}, Memory usage: {} bytes'
      .format(batch[0].shape, batch[0].dtype, batch[0].element_size()*batch[0].nelement()))
print('Y train batch, shape: {}, data type: {}, Memory usage: {} bytes'
      .format(batch[1].shape, batch[1].dtype, batch[1].element_size()*batch[1].nelement()))

imgs, targets = batch

X train batch, shape: torch.Size([16, 3, 224, 224]), data type: torch.float32, Memory usage: 9633792 bytes
Y train batch, shape: torch.Size([16]), data type: torch.int64, Memory usage: 128 bytes
CPU times: user 5.03 s, sys: 57.8 ms, total: 5.09 s
Wall time: 5.59 s


batch_size = 16
W = image_size
H = image_size


lam = torch.rand(batch_size)
s_index = torch.randperm(batch_size)      # Shuffle index
rand_x = torch.randint(W, (batch_size,))
rand_y = torch.randint(H, (batch_size,))
cut_rat = torch.sqrt(1. - lam) ## cut ratio according to the random lambda

x1 = torch.clip(rand_x - rand_x / 2, min=0).long()
x2 = torch.clip(rand_x + rand_x / 2, max=W-1).long()
y1 = torch.clip(rand_y - rand_y / 2, min=0).long()
y2 = torch.clip(rand_y + rand_y / 2, max=H-1).long()

mask_ext, mask_int = cut_mask(x1, x2, y1, y2, batch_size, W, H)


# vérifier si le masque intérieur et l'image ont le même nombre de dimensions
try:
    assert imgs.dim() == mask_int.dim()
    print('OK!')
except:
    print(f'Mismatch: \n dim imgs = {imgs.dim()} \n dim mask int = {mask_int.dim()} ')

OK!


# vérifier si le masque extérieur et l'image ont le même nombre de dimensions
try:
    assert imgs.dim() == mask_ext.dim()
    print('OK!')
except:
    print(f'Mismatch: \n dim imgs = {imgs.dim()} \n dim mask ext = {mask_ext.dim()} ')

OK!


imgs = mask_ext * imgs + mask_int * imgs[s_index, :]


for i in range(4):
    img = imgs[i].numpy().transpose((1,2,0))
    plt.imshow(img)
    plt.axis('off')
    plt.show()


command = f'dlojz_da_3.py -b {bs_optim} --image-size {image_size} --test'
n_gpu = 1
jobid = gpu_jobs_submitter(command, n_gpu, MODULE, name=name,
                    account=account, time_max='00:10:00')
print(f'jobid = {jobid}')

batch job 0: 1 GPUs distributed on 1 nodes with 1 tasks / 1 gpus per node and 8 cpus per task
Submitted batch job 888142
jobid = ['888142']


jobid = ['888142']


display_slurm_queue(name)

             JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)
            888142    gpu_p5     tutu  ssos938  R       2:22      1 jean-zay-iam05

 Done!


controle_technique(jobid)

Train throughput: 2266.13 images/second
GPU throughput: 2567.43 images/second
epoch time: 565.52 seconds
-----------
training step time average (fwd/bkwd on GPU): 0.199421 sec (6.7%/105.6%) +/- 0.077219
loading step time average (IO + CPU to GPU transfer): 0.026515 sec +/- 0.049210


turbo_profiler(jobid)

>>> Turbo Profiler >>> Training complete in 42.835168 s

DLO-JZ Data Augmentation - Jour 4¶

Objet du notebook¶

Environnement de calcul¶

Gestion de la queue SLURM¶

Debug¶

Différence entre deux scripts¶

Garage - Mise à niveau¶

TP_DA_1 : RandAugment¶

Transformation RandAugment sur CPU¶

TP_DA_2 : mixup¶

Transformation mixup sur CPU¶

Transformation mixup sur GPU¶

TP_DA_3 : CutMix¶

Transformation CutMix sur GPU¶

Optimisation de la transformation CutMix¶

Test de la fonction implémentée¶

Intégration de la nouvelle version dans cutmix.py¶